#!/usr/bin/python3
# -*- coding: utf-8 -*-
"""
@author:  runyuanye
@contact: runyuanye@163.com
"""

import os
import sys
import time
import shutil
import numpy as np
import math
from matplotlib import pyplot as plt
from math import sin, cos, asin, sqrt
import random

sys.path.append('.')

"""
   Test
"""


def file_rename():
    start_idxs = [
        0,
        2283,
        2283,
    ]
    image_paths = [
        r'/rootfs/media/kasim/Data/data/ErisedVideoFaceReID/train',
        r'/rootfs/media/kasim/Data/data/ErisedVideoFaceReID/query',
        r'/rootfs/media/kasim/Data/data/ErisedVideoFaceReID/gallery',
    ]
    file_list_paths = [
        r'/rootfs/media/kasim/Data/data/ErisedVideoFaceReID/train.txt',
        r'/rootfs/media/kasim/Data/data/ErisedVideoFaceReID/query.txt',
        r'/rootfs/media/kasim/Data/data/ErisedVideoFaceReID/gallery.txt'
    ]

    out_path = r'/rootfs/media/kasim/Data/data/ErisedVideoFaceReID/ErisedVideoFaceRec'

    if not os.path.exists(out_path):
        os.makedirs(out_path)

    out_file_list_path = os.path.join(out_path, 'face_list.txt')

    faceid_files_map = {}
    for start_idx, image_path, file_list_path in zip(start_idxs, image_paths, file_list_paths):
        with open(file_list_path, 'r') as file:
            for line in file.readlines():
                # 0001/0001_F0009B00.jpg
                src_file_path = line.strip()
                lines = src_file_path.split('/')
                faceid = int(lines[0]) + start_idx
                src_file_path = os.path.join(image_path, src_file_path)
                files = faceid_files_map.get(faceid, None)
                if files is None:
                    faceid_files_map[faceid] = [src_file_path]
                else:
                    files.append(src_file_path)

    faceid_count = 0
    file_count = 0
    with open(out_file_list_path, 'w') as file:
        faceids = list(faceid_files_map.keys())
        faceids.sort()
        for faceid in faceids:
            faceid_count += 1
            files = faceid_files_map[faceid]
            for frame_num, src_file_path in enumerate(files):
                file_count += 1
                if file_count % 100 == 0:
                    print('Process File Count:', file_count)

                dst_file_path = '{:04d}/{:04d}_F{:04d}B00.jpg'.format(faceid, faceid, frame_num)
                file.write(dst_file_path + '\n')

                dst_file_path = os.path.join(out_path, dst_file_path)
                if src_file_path == dst_file_path:
                    continue
                dst_dir = os.path.dirname(dst_file_path)
                if not os.path.exists(dst_dir):
                    os.makedirs(dst_dir)
                shutil.copy(src_file_path, dst_file_path)
        print('Process File Count:', file_count)

    os.system('chmod a+wr {} -Rf'.format(out_path))
    os.system('chmod a+wr {}'.format(out_file_list_path))
    print('Face Count:', faceid_count, 'Face Image Count:', file_count)

    print('Finish!')


def file_rename2():
    image_paths = [
        r'/rootfs/media/kasim/Data1/data/FaceBaidu',
    ]
    file_list_paths = [
        r'/rootfs/media/kasim/Data1/data/FaceBaidu/ReplaceFaces.txt',
    ]

    out_path = r'/rootfs/media/kasim/Data1/data/ErisedFaceRec'

    if not os.path.exists(out_path):
        os.makedirs(out_path)

    out_file_list_path = os.path.join(out_path, 'face_list.txt')

    faceid_files_map = {}
    faceid_map = {}
    start_idx = 1
    for image_path, file_list_path in zip(image_paths, file_list_paths):
        with open(file_list_path, 'r') as file:
            for line in file.readlines():
                # e050522f89fb3f74aea381765c12ce76 ec99fb9a85f33afba953c0bc270e9aab/237ada5f1f3bcf3951867632b72eaa15.jpg 17,27,118,153,1,1032
                lines = line.strip().split()
                faceid_str = lines[0]
                src_file_path = lines[1]

                faceid = faceid_map.get(faceid_str, None)
                if faceid is None:
                    faceid_map[faceid_str] = start_idx
                    faceid = start_idx
                    start_idx += 1
                src_file_path = os.path.join(image_path, src_file_path)
                files = faceid_files_map.get(faceid, None)
                if files is None:
                    faceid_files_map[faceid] = [src_file_path]
                else:
                    files.append(src_file_path)

    faceid_count = 0
    file_count = 0
    with open(out_file_list_path, 'w') as file:
        faceids = list(faceid_files_map.keys())
        faceids.sort()
        for faceid in faceids:
            faceid_count += 1
            files = faceid_files_map[faceid]
            for frame_num, src_file_path in enumerate(files):
                file_count += 1
                if file_count % 100 == 0:
                    print('Process File Count:', file_count)

                dst_file_path = '{:04d}/{:04d}_F{:04d}B00.jpg'.format(faceid, faceid, frame_num)
                file.write(dst_file_path + '\n')

                dst_file_path = os.path.join(out_path, dst_file_path)
                if src_file_path == dst_file_path:
                    continue
                dst_dir = os.path.dirname(dst_file_path)
                if not os.path.exists(dst_dir):
                    os.makedirs(dst_dir)
                shutil.copy(src_file_path, dst_file_path)
        print('Process File Count:', file_count)

    os.system('chmod a+wr {} -Rf'.format(out_path))
    os.system('chmod a+wr {}'.format(out_file_list_path))
    print('Face Count:', faceid_count, 'Face Image Count:', file_count)

    print('Finish!')


def draw_hist(plt, name, xlist, bins):
    if isinstance(xlist, np.ndarray):
        x = xlist
    else:
        x = np.array(xlist)
    n, bins, patches = plt.hist(x, bins, align=u'mid', facecolor='blue', alpha=0.5)
    plt.title('{}: {:.2f} {:.3f} {:.2f} {:.2f}'.format(name, x.mean(), x.std(), x.min(), x.max()))


def show_hist(value_list, min_value=0.0, max_value=1.0, dbin=0.1, dxticks=0.1, title_str='Age'):
    bins = np.linspace(min_value, max_value, int((max_value - min_value) / dbin) + 1)
    xticks = np.linspace(min_value, max_value, int((max_value - min_value) / dxticks) + 1)

    fig = plt.figure(title_str, figsize=(24, 12))
    plt.subplot(111)
    draw_hist(plt, title_str, value_list, bins)
    plt.xticks(xticks)
    plt.tight_layout()
    plt.show()


def show_hist_ex(value_list, bin_count=20, scale=1.0, title_str='Age'):
    if isinstance(value_list, np.ndarray):
        values = value_list
    else:
        values = np.array(value_list)
    if scale != 1.0:
        values = values * scale
    min_value = values.min()
    max_value = values.max()
    min_value = math.floor(min_value)
    max_value = math.ceil(max_value)

    diff_value = max_value - min_value
    if (bin_count > diff_value) or (bin_count is None) or (bin_count <= 0):
        bin_count = diff_value
    dxticks = diff_value / bin_count

    show_hist(values, min_value, max_value, dxticks, dxticks, title_str)


def face_rec_video():
    start_idxs = [
        0,
        2283,
        2283,
    ]
    image_paths = [
        r'/rootfs/media/kasim/Data/data/ErisedVideoFaceReID/train',
        r'/rootfs/media/kasim/Data/data/ErisedVideoFaceReID/query',
        r'/rootfs/media/kasim/Data/data/ErisedVideoFaceReID/gallery',
    ]
    file_list_paths = [
        r'/rootfs/media/kasim/Data/data/ErisedVideoFaceReID/train_pair_files.txt',
        r'/rootfs/media/kasim/Data/data/ErisedVideoFaceReID/query_pair_files.txt',
        r'/rootfs/media/kasim/Data/data/ErisedVideoFaceReID/gallery_pair_files.txt'
    ]

    out_path = r'/rootfs/media/kasim/Data/data/ErisedVideoFaceReID'

    if not os.path.exists(out_path):
        os.makedirs(out_path)

    out_file_list_path = os.path.join(out_path, 'face_rec_video_list.txt')

    faceid_files_map = {}
    for start_idx, image_path, file_list_path in zip(start_idxs, image_paths, file_list_paths):
        with open(file_list_path, 'r') as file:
            for line in file.readlines():
                # gd/zs/sqqjd/ycjy/150100414a54443452064d2742c43600/video/20190924_092827/000083_000.jpg:0001_F0000B00.jpg
                lines = line.strip().split(':')
                video_path = os.path.dirname(lines[0])
                faceid = int(lines[1].split('_')[0]) + start_idx
                files = faceid_files_map.get(faceid, None)
                if files is None:
                    faceid_files_map[faceid] = {video_path}
                else:
                    files.add(video_path)

    faceid_count = 0
    file_count = 0
    video_counts = []
    with open(out_file_list_path, 'w') as file:
        faceids = list(faceid_files_map.keys())
        faceids.sort()
        for faceid in faceids:
            files = faceid_files_map[faceid]
            video_count = len(files)
            if video_count <= 1:
                # print(faceid)
                continue
            faceid_count += 1
            video_counts.append(video_count)
            for video_num, src_file_path in enumerate(files):
                file_count += 1
                # if file_count % 100 == 0:
                #     print('Process File Count:', file_count)

                out_info = '{:04d} {}.avi\n'.format(faceid_count, src_file_path)
                file.write(out_info)
        # print('Process File Count:', file_count)

    os.system('chmod a+wr {}'.format(out_path))
    os.system('chmod a+wr {}'.format(out_file_list_path))
    print('Face Count:', faceid_count, 'Face Video Count:', file_count)

    show_hist_ex(video_counts, bin_count=-1, scale=1.0, title_str='Face Video Count')

    print('Finish!')


def estimate_gaze_from_headpose(yaw, pitch, roll, gaze_thr=15):
    """
    Estimate whether a person is gazing the screen by his/her head pose
    :param yaw: Yaw of head pose in angular degree
    :param pitch: Pitch of head pose in angular degree
    :param roll: Roll of head pose in angular degree
    """
    # Convert from angular to radian degree
    pitch = pitch * np.pi / 180
    yaw = -(yaw * np.pi / 180)
    # roll = roll * np.pi / 180

    # Angular between Z-Axis (out of the screen) and
    x_projected = sin(yaw)
    y_projected = -cos(yaw) * sin(pitch)

    #what the z_angle present???
    z_angle = asin(sqrt(x_projected ** 2+y_projected ** 2))

    # Using a fixed angular threshold for now
    if z_angle * 180 / np.pi > gaze_thr:
        return False
    else:
        return True


def filter_gaze_video():
    GAZE_COUNT = 1
    GAZE_THR = 20
    ANGLE_SCORE = 0.1
    ANGLE_DIFF = 20
    face_attr_list_file = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/extract_image_list_72.attr.txt'
    face_rec_video_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/face_rec_video_list.txt'

    faceid_video_path_dict = {}
    video_path_gaze_dict = {}
    with open(face_rec_video_path, 'r') as file:
        for line in file.readlines():
            lines = line.strip().split()
            faceid = int(lines[0])
            video_path = os.path.splitext(lines[1])[0]
            video_paths = faceid_video_path_dict.get(faceid, None)
            if video_paths is None:
                faceid_video_path_dict[faceid] = {video_path}
            else:
                video_paths.add(video_path)
            video_path_gaze_dict[video_path] = 0

    with open(face_attr_list_file, 'r') as file:
        isfirst = True
        for line in file.readlines():
            # file_name;bin_pitch;bin_roll;bin_yaw;pitch;roll;yaw;quality;mask
            # gd/zs/nqjd/dxjyjy/150100414a54443452064d2742c93600/video/20190817_030936/000000_000.jpg;-2.651,0.516;-2.601,0.566;-89.733,0.318;-1.642;-5.795;-31.306;0.045,0.150,0.421,0.384;72.692,73.769,63.882,65.441,77.657,95.700,72.600,64.765,65.577
            lines = line.strip().split(';')
            if isfirst:
                isfirst = False
                continue
            bin_pitch = lines[1].split(',')
            bin_roll = lines[2].split(',')
            bin_yaw = lines[3].split(',')
            if (float(bin_pitch[1]) < ANGLE_SCORE) or (float(bin_roll[1]) < ANGLE_SCORE) or (float(bin_yaw[1]) < ANGLE_SCORE):
                continue
            file_name = os.path.dirname(lines[0])
            gaze_count = video_path_gaze_dict.get(file_name, None)
            if gaze_count is None:
                continue
            pitch = float(lines[4])
            roll = float(lines[5])
            yaw = float(lines[6])
            if (abs(float(bin_pitch[0]) - pitch) > ANGLE_DIFF) or (abs(float(bin_roll[0]) - roll) > ANGLE_DIFF) or (abs(float(bin_yaw[0]) - yaw) > ANGLE_DIFF):
                continue
            gaze = estimate_gaze_from_headpose(yaw, pitch, roll, gaze_thr=GAZE_THR)
            if gaze:
                video_path_gaze_dict[file_name] = gaze_count + 1

    gaze_counts = []
    out_path = os.path.join(os.path.dirname(face_rec_video_path), 'face_rec_video_gaze_list.txt')
    with open(out_path, 'w') as file:
        faceids = list(faceid_video_path_dict.keys())
        faceids.sort()
        faceid_count = 0
        file_count = 0
        for faceid in faceids:
            video_paths = faceid_video_path_dict[faceid]
            video_path_gazes = []
            for video_path in video_paths:
                gaze_count = video_path_gaze_dict[video_path]
                if gaze_count < GAZE_COUNT:
                    print(video_path)
                    continue
                gaze_counts.append(gaze_count)
                video_path_gazes.append([video_path, gaze_count])
            video_count = len(video_path_gazes)
            if video_count >= 2:
                faceid_count += 1
                file_count += video_count
                for video_path_gaze in video_path_gazes:
                    video_path, gaze_count = video_path_gaze
                    out_info = '{:04d} {}.avi {}\n'.format(faceid_count, video_path, gaze_count)
                    file.write(out_info)
    os.system('chmod a+wr {}'.format(out_path))
    print('Face Count:', faceid_count, 'Face Video Count:', file_count)

    show_hist_ex(gaze_counts, bin_count=100, scale=1.0, title_str='Face Video Gaze Count')

    print('Finish!')


def filter_gaze_file():
    face_rec_video_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/face_rec_video_gaze_list.txt'
    filter_file_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/video_one_face.txt'
    out_video_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/filter_face_rec_video_gaze_list.txt'

    filter_files = set()
    with open(filter_file_path, 'r') as file:
        for line in file.readlines():
            filter_files.add(line.strip())

    filter_count = 0
    save_count = 0
    with open(out_video_path, 'w') as out_file:
        with open(face_rec_video_path, 'r') as file:
            for line in file.readlines():
                lines = line.strip().split()
                video_path = lines[1]
                if video_path not in filter_files:
                    filter_count += 1
                    continue
                out_file.write(line)
                save_count += 1
    os.system('chmod a+wr {}'.format(out_video_path))
    print('filter count:', filter_count, ', keep count:', save_count)


def split_face_rec_video():
    MAX_VIDEO_COUNT = 20
    face_rec_video_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/filter_face_rec_video_gaze_list.txt'
    random.seed(123456)

    faceid_video_path_dict = {}
    video_path_gaze_dict = {}
    with open(face_rec_video_path, 'r') as file:
        for line in file.readlines():
            lines = line.strip().split()
            faceid = int(lines[0])
            video_path = lines[1]
            video_paths = faceid_video_path_dict.get(faceid, None)
            if video_paths is None:
                faceid_video_path_dict[faceid] = [video_path]
            else:
                video_paths.append(video_path)
            video_path_gaze_dict[video_path] = int(lines[2])

    video_counts = []
    video_counts1 = []
    faceids = list(faceid_video_path_dict.keys())
    faceids.sort()
    faceid_count = 0
    gallery_file_count = 0
    query_file_count = 0
    query_file_path = os.path.join(os.path.dirname(face_rec_video_path), 'face_rec_query_video_list.txt')
    gallery_file_path = os.path.join(os.path.dirname(face_rec_video_path), 'face_rec_gallery_video_list.txt')
    query_file = open(query_file_path, 'w')
    gallery_file = open(gallery_file_path, 'w')
    for faceid in faceids:
        video_paths = faceid_video_path_dict[faceid]
        video_count = len(video_paths)
        video_counts.append(video_count)
        random.shuffle(video_paths)
        video_dates = set()
        new_video_paths = []
        remain_video_paths = {}
        for video_path in video_paths:
            video_date = os.path.basename(video_path).split('_')[0]
            # video_date = time.strptime(video_date, '%Y%m%d_%H%M%S.avi')
            # # video_date = time.mktime(video_date) - 3600 * 8  # 北京时间
            # video_date = time.mktime(video_date)

            if video_date in video_dates:
                remain_video_path = remain_video_paths.get(video_date, None)
                if remain_video_path is None:
                    remain_video_paths[video_date] = [video_path]
                else:
                    remain_video_path.append(video_path)
                continue
            video_dates.add(video_date)
            new_video_paths.append(video_path)

        video_count = len(new_video_paths)
        if video_count < 2:
            continue
        faceid_count += 1

        if video_count > MAX_VIDEO_COUNT:
            video_count = MAX_VIDEO_COUNT

        gallery_count = video_count // 2
        gallery_paths = new_video_paths[0:gallery_count]
        query_paths = new_video_paths[gallery_count:video_count]
        query_count = len(query_paths)

        _query_file_count = 0
        _gallery_file_count = 0
        query_remain_video_paths = []
        for query_path in query_paths:
            out_info = '{:04d} {}\n'.format(faceid_count, query_path)
            query_file.write(out_info)
            _query_file_count += 1
            video_date = os.path.basename(query_path).split('_')[0]
            if video_date in remain_video_paths.keys():
                query_remain_video_paths.extend(remain_video_paths[video_date])

        gallery_remain_video_paths = []
        for gallery_path in gallery_paths:
            out_info = '{:04d} {}\n'.format(faceid_count, gallery_path)
            gallery_file.write(out_info)
            _gallery_file_count += 1
            video_date = os.path.basename(gallery_path).split('_')[0]
            if video_date in remain_video_paths.keys():
                gallery_remain_video_paths.extend(remain_video_paths[video_date])

        remain_gallery_count = (MAX_VIDEO_COUNT // 2) - gallery_count
        gallery_remain_video_paths_count = len(gallery_remain_video_paths)
        if remain_gallery_count > gallery_remain_video_paths_count:
            remain_gallery_count = gallery_remain_video_paths_count
        if remain_gallery_count > 0:
            random.shuffle(gallery_remain_video_paths)
            gallery_remain_video_paths = gallery_remain_video_paths[0:remain_gallery_count]
            for gallery_path in gallery_remain_video_paths:
                out_info = '{:04d} {}\n'.format(faceid_count, gallery_path)
                gallery_file.write(out_info)
                _gallery_file_count += 1

        remain_query_count = (MAX_VIDEO_COUNT // 2) - query_count
        query_remain_video_paths_count = len(query_remain_video_paths)
        if remain_query_count > query_remain_video_paths_count:
            remain_query_count = query_remain_video_paths_count
        if remain_query_count > 0:
            random.shuffle(query_remain_video_paths)
            query_remain_video_paths = query_remain_video_paths[0:remain_query_count]
            for query_path in query_remain_video_paths:
                out_info = '{:04d} {}\n'.format(faceid_count, query_path)
                query_file.write(out_info)
                _query_file_count += 1

        # for query_path in query_paths:
        #     out_info = '{:04d} {}\n'.format(faceid_count, query_path)
        #     query_file.write(out_info)
        #     _query_file_count += 1
        # for gallery_path in gallery_paths:
        #     out_info = '{:04d} {}\n'.format(faceid_count, gallery_path)
        #     gallery_file.write(out_info)
        #     _gallery_file_count += 1
        video_counts1.append(_gallery_file_count+_query_file_count)
        gallery_file_count += _gallery_file_count
        query_file_count += _query_file_count

    query_file.close()
    gallery_file.close()
    os.system('chmod a+wr {}'.format(query_file_path))
    os.system('chmod a+wr {}'.format(gallery_file_path))
    print('Face Count:', faceid_count, ', Face Query Video Count:', query_file_count, ', Face Gallery Video Count:', gallery_file_count, ', Face Video Count:', (query_file_count+gallery_file_count))

    show_hist_ex(video_counts, bin_count=-1, scale=1.0, title_str='Face Video Count 0')
    show_hist_ex(video_counts1, bin_count=-1, scale=1.0, title_str='Face Video Count 1')

    print('Finish!')


def cal_file_count():
    file_path = r'/rootfs/media/kasim/DataSet/DataSet/VideoFaceImage/extract_image_list_72.txt'
    video_files = set()
    with open(file_path, 'r') as file:
        # gd/zs/nqjd/dxjyjy/150100414a54443452064d2742c93600/video/20190817_030936/000000_000.jpg 11,22,85,132
        for line in file.readlines():
            image_file_name = line.split()[0]
            video_file_name = os.path.dirname(image_file_name)
            video_files.add(video_file_name)
    print('Video File Count:', len(video_files))


def copy_video_face_file():
    video_file_lists = [
        r'/rootfs/media/kasim/Data/data/ErisedVideoFaceReID/face_rec_query_video_list.txt',
        r'/rootfs/media/kasim/Data/data/ErisedVideoFaceReID/face_rec_gallery_video_list.txt'
    ]
    video_base_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage'
    out_video_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImageSelect'
    if not os.path.exists(out_video_path):
        os.makedirs(out_video_path)
        os.system('chmod a+wr {}'.format(out_video_path))
    for video_file_list in video_file_lists:
        with open(video_file_list, 'r') as file:
            for line in file.readlines():
                video_path = os.path.splitext(line.strip().split()[1])[0]
                src_path = os.path.join(video_base_path, video_path)
                if not os.path.exists(src_path):
                     continue
                dst_path = os.path.join(out_video_path, video_path)
                shutil.move(src_path, dst_path)


def check_video_face_file():
    video_file_lists = [
        r'/rootfs/media/kasim/Data/data/ErisedVideoFaceReID/face_rec_query_video_list.txt',
        r'/rootfs/media/kasim/Data/data/ErisedVideoFaceReID/face_rec_gallery_video_list.txt'
    ]
    video_base_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImageSelect'
    out_video_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/no_video_list.txt'
    out_file = open(out_video_path, 'w')
    for video_file_list in video_file_lists:
        with open(video_file_list, 'r') as file:
            for line in file.readlines():
                video_path = os.path.splitext(line.strip().split()[1])[0]
                src_path = os.path.join(video_base_path, video_path)
                if not os.path.exists(src_path):
                    out_file.write(video_path + '\n')
    out_file.close()
    os.system('chmod a+wr {}'.format(out_video_path))


def cal_file_face_count():
    file_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/extract_image_list_72.txt'
    video_file_frame_face_counts = {}
    with open(file_path, 'r') as file:
        # gd/zs/nqjd/dxjyjy/150100414a54443452064d2742c93600/video/20190817_030936/000000_000.jpg 11,22,85,132
        for line in file.readlines():
            image_file_name = line.split()[0]
            video_file_name = os.path.dirname(image_file_name)
            image_file_name = os.path.basename(image_file_name)
            frame_id = int(image_file_name.split('_')[0])
            frame_face_counts = video_file_frame_face_counts.get(video_file_name, None)
            if frame_face_counts is None:
                video_file_frame_face_counts[video_file_name] = {frame_id: 1}
            else:
                face_count = frame_face_counts.get(frame_id, None)
                if face_count is None:
                    frame_face_counts[frame_id] = 1
                else:
                    frame_face_counts[frame_id] = face_count + 1
    video_file_face_counts = {}
    for video_file_name, frame_face_counts in video_file_frame_face_counts.items():
        if len(frame_face_counts) > 0:
            face_count = max(frame_face_counts.values())
        else:
            face_count = 0
        video_file_face_counts[video_file_name] = face_count

    out_video_face_count_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/video_face_count.txt'
    out_video_one_face_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/video_one_face.txt'
    with open(out_video_face_count_path, 'w') as file:
        with open(out_video_one_face_path, 'w') as onefile:
            video_files = list(video_file_face_counts.keys())
            video_files.sort()
            for video_file in video_files:
                face_count = video_file_face_counts[video_file]
                out_info = '{}.avi {}\n'.format(video_file, face_count)
                file.write(out_info)
                if face_count == 1:
                    out_info = '{}.avi\n'.format(video_file)
                    onefile.write(out_info)
    os.system('chmod a+wr {}'.format(out_video_face_count_path))
    os.system('chmod a+wr {}'.format(out_video_one_face_path))


def filter_file_list():
    video_file_lists = [
        r'/rootfs/media/kasim/Data1/data/VideoFaceImage/face_rec_query_video_list.txt',
        r'/rootfs/media/kasim/Data1/data/VideoFaceImage/face_rec_gallery_video_list.txt'
    ]

    # filter_video_file_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/filter_file_list.txt'
    filter_video_file_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/merge_filter_file_list.txt'
    video_file_images = {}
    with open(filter_video_file_path, 'r') as file:
        for line in file.readlines():
            line = line.strip()
            video_path = os.path.dirname(line)
            image_path = os.path.basename(line)
            images = video_file_images.get(video_path, None)
            if images is None:
                video_file_images[video_path] = [image_path]
            else:
                images.append(image_path)

    faceid_video_files = {}
    for video_file_list in video_file_lists:
        with open(video_file_list, 'r') as file:
            for line in file.readlines():
                lines = line.strip().split()
                faceid = lines[0]
                video_path = os.path.splitext(lines[1])[0]
                video_files = faceid_video_files.get(faceid, None)
                if video_files is None:
                    faceid_video_files[faceid] = [video_path]
                else:
                    video_files.append(video_path)

    out_video_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/filter_face_rec_video_list.txt'
    with open(out_video_path, 'w') as file:
        video_file_keys = video_file_images.keys()
        face_count = 1
        has_image = False
        file_count = 0
        for faceid, video_files in faceid_video_files.items():
            video_files.sort()
            for video_file in video_files:
                if video_file not in video_file_keys:
                    continue
                images = video_file_images[video_file]
                if len(images) > 0:
                    file_count += 1
                    has_image = True
                for image in images:
                    out_info = '{} {}/{}\n'.format(face_count, video_file, image)
                    file.write(out_info)
            if has_image:
                face_count += 1
    os.system('chmod a+wr {}'.format(out_video_path))
    print('Face Count:', (face_count-1), ', File Count:', file_count)


def filter_attr_video():
    ANGLE_SCORE = 0.1
    ANGLE_DIFF = 20

    MIN_PITCH1 = -20.0
    MAX_PITCH1 = 20.0
    MIN_YAW1 = -20.0
    MAX_YAW1 = 20.0
    MIN_ROLL1 = -30.0
    MAX_ROLL1 = 30.0

    MIN_PITCH2 = -30.0
    MAX_PITCH2 = 30.0
    MIN_YAW2 = -30.0
    MAX_YAW2 = 30.0
    MIN_ROLL2 = -45.0
    MAX_ROLL2 = 45.0

    face_attr_list_file = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/extract_image_list_72.attr.txt'
    face_rec_video_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/filter_face_rec_video_list.txt'

    faceid_videos_dict = {}
    image_faceids_set = set()
    with open(face_rec_video_path, 'r') as file:
        for line in file.readlines():
            # 1 gd/zs/sqqjd/dxssjyeq/150100414a54443452064d2742c73600/video/20190830_133551/000000_000.jpg
            lines = line.strip().split()
            faceid = int(lines[0])
            image_path = lines[1]
            video_paths = faceid_videos_dict.get(faceid, None)
            video_name = os.path.dirname(image_path)
            if video_paths is None:
                faceid_videos_dict[faceid] = {video_name}
            else:
                video_paths.add(video_name)
            image_faceids_set.add(image_path)

    video_image_attrs = {}
    file_count = 0
    with open(face_attr_list_file, 'r') as file:
        isfirst = True
        for line in file.readlines():
            # file_name;bin_pitch;bin_roll;bin_yaw;pitch;roll;yaw;quality;mask
            # gd/zs/nqjd/dxjyjy/150100414a54443452064d2742c93600/video/20190817_030936/000000_000.jpg;-2.651,0.516;-2.601,0.566;-89.733,0.318;-1.642;-5.795;-31.306;0.045,0.150,0.421,0.384;72.692,73.769,63.882,65.441,77.657,95.700,72.600,64.765,65.577
            lines = line.strip().split(';')
            if isfirst:
                isfirst = False
                continue
            file_name = lines[0]
            if file_name not in image_faceids_set:
                continue
            bin_pitch = lines[1].split(',')
            bin_roll = lines[2].split(',')
            bin_yaw = lines[3].split(',')
            bin_pitch_score = float(bin_pitch[1])
            bin_roll_score = float(bin_roll[1])
            bin_yaw_score = float(bin_yaw[1])
            bin_pitch = float(bin_pitch[0])
            bin_roll = float(bin_roll[0])
            bin_yaw = float(bin_yaw[0])

            pitch = float(lines[4])
            roll = float(lines[5])
            yaw = float(lines[6])

            quality = lines[7].split(',')
            quality = 0.3333333333 * float(quality[2]) + 0.66666666 * float(quality[1]) + float(quality[0])
            video_name = os.path.dirname(file_name)
            file_name = os.path.basename(file_name)
            image_attrs = video_image_attrs.get(video_name, None)
            if image_attrs is None:
                video_image_attrs[video_name] = {'file_name': [file_name], 'pitch': [pitch], 'roll': [roll], 'yaw': [yaw], 'bin_pitch': [bin_pitch], 'bin_roll': [bin_roll], 'bin_yaw': [bin_yaw], 'bin_pitch_score': [bin_pitch_score], 'bin_roll_score': [bin_roll_score], 'bin_yaw_score': [bin_yaw_score], 'quality': [quality]}
            else:
                image_attrs['file_name'].append(file_name)
                image_attrs['pitch'].append(pitch)
                image_attrs['roll'].append(roll)
                image_attrs['yaw'].append(yaw)
                image_attrs['bin_pitch'].append(bin_pitch)
                image_attrs['bin_roll'].append(bin_roll)
                image_attrs['bin_yaw'].append(bin_yaw)
                image_attrs['bin_pitch_score'].append(bin_pitch_score)
                image_attrs['bin_roll_score'].append(bin_roll_score)
                image_attrs['bin_yaw_score'].append(bin_yaw_score)
                image_attrs['quality'].append(quality)
            file_count += 1
            # if file_count > 1000:
            #     break

    out_path = os.path.join(os.path.dirname(face_rec_video_path), 'typical_face_rec_video_list.txt')
    with open(out_path, 'w') as file:
        video_typical_image_dict = {}
        for video_name, image_attrs in video_image_attrs.items():
            file_names = image_attrs['file_name']
            pitch = np.array(image_attrs['pitch'], dtype=np.float32)
            roll = np.array(image_attrs['roll'], dtype=np.float32)
            yaw = np.array(image_attrs['yaw'], dtype=np.float32)
            bin_pitch = np.array(image_attrs['bin_pitch'], dtype=np.float32)
            bin_roll = np.array(image_attrs['bin_roll'], dtype=np.float32)
            bin_yaw = np.array(image_attrs['bin_yaw'], dtype=np.float32)
            bin_pitch_score = np.array(image_attrs['bin_pitch_score'], dtype=np.float32)
            bin_roll_score = np.array(image_attrs['bin_roll_score'], dtype=np.float32)
            bin_yaw_score = np.array(image_attrs['bin_yaw_score'], dtype=np.float32)
            quality = np.array(image_attrs['quality'], dtype=np.float32)

            pitch_diff = np.abs(bin_pitch - pitch)
            roll_diff = np.abs(bin_roll - roll)
            yaw_diff = np.abs(bin_yaw - yaw)

            score_mask = (bin_pitch_score > ANGLE_SCORE) & (bin_roll_score > ANGLE_SCORE) & (bin_yaw_score > ANGLE_SCORE)
            diff_mask = (pitch_diff < ANGLE_DIFF) & (roll_diff < ANGLE_DIFF) & (yaw_diff < ANGLE_DIFF)
            mask = score_mask & diff_mask
            if mask.any():
                pitch = pitch[mask]
                roll = roll[mask]
                yaw = yaw[mask]
                quality = quality[mask]
                file_names = [f for m, f in zip(mask, file_names) if m]
            idxes = np.arange(0, len(file_names))
            pitch_mask = np.logical_and(MIN_PITCH1 <= pitch, pitch <= MAX_PITCH1)
            roll_mask = np.logical_and(MIN_ROLL1 <= roll, roll <= MAX_ROLL1)
            yaw_mask = np.logical_and(MIN_YAW1 <= yaw, yaw <= MAX_YAW1)
            mask = pitch_mask & roll_mask & yaw_mask
            if mask.any():
                max_quality_idx = quality[mask].argmax()
                idx = idxes[mask][max_quality_idx]
            else:
                pitch_mask = np.logical_and(MIN_PITCH2 <= pitch, pitch <= MAX_PITCH2)
                roll_mask = np.logical_and(MIN_ROLL2 <= roll, roll <= MAX_ROLL2)
                yaw_mask = np.logical_and(MIN_YAW2 <= yaw, yaw <= MAX_YAW2)
                mask = pitch_mask & roll_mask & yaw_mask
                if mask.any():
                    max_quality_idx = quality[mask].argmax()
                    idx = idxes[mask][max_quality_idx]
                else:
                    idx = quality.argmax()
            file_name = file_names[idx]
            video_typical_image_dict[video_name] = file_name

        faceids = list(faceid_videos_dict.keys())
        faceids.sort()
        for faceid in faceids:
            video_names = faceid_videos_dict[faceid]
            video_names = list(video_names)
            video_names.sort()
            for video_name in video_names:
                image_name = video_typical_image_dict.get(video_name, None)
                if image_name is None:
                    continue
                out_info = '{} {} {}\n'.format(faceid, video_name, image_name)
                file.write(out_info)

    os.system('chmod a+wr {}'.format(out_path))

    print('Finish!')


def image_to_video_list():
    file_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/file_list.txt'
    video_files = set()
    with open(file_path, 'r') as file:
        for line in file.readlines():
            # gd/zs/dqjd/xgly/150100414a5444345203bcd04294b500/video/20190912_155158/000000_000.jpg
            video_file_name = os.path.dirname(line.strip())
            video_files.add(video_file_name)
    out_file_path = os.path.join(os.path.dirname(file_path), 'video_list.txt')
    with open(out_file_path, 'w') as file:
        for video_file in video_files:
            out_info = '{}\n'.format(video_file)
            file.write(out_info)
    print('Video File Count:', len(video_files))


def query_gallery_to_video_list():
    video_file_lists = [
        r'/rootfs/media/kasim/Data1/data/VideoFaceImage/face_rec_query_video_list.txt',
        r'/rootfs/media/kasim/Data1/data/VideoFaceImage/face_rec_gallery_video_list.txt'
    ]
    out_video_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/query_gallery_video_list.txt'
    with open(out_video_path, 'w') as out_file:
        for video_file_list in video_file_lists:
            with open(video_file_list, 'r') as file:
                for line in file.readlines():
                    video_path = os.path.splitext(line.strip().split()[1])[0]
                    out_info = '{}\n'.format(video_path)
                    out_file.write(out_info)
    os.system('chmod a+wr {}'.format(out_video_path))


def filter_video_list():
    file_list_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/query_gallery_video_list.txt'
    filter_file_list_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/video_list.txt'
    out_file_list_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/filter_video_list.txt'

    filter_file_set = set()
    with open(filter_file_list_path, 'r') as file:
        for line in file.readlines():
            filter_file_set.add(line)

    with open(out_file_list_path, 'w') as out_file:
        with open(file_list_path, 'r') as file:
            for line in file.readlines():
                if line in filter_file_set:
                    continue
                out_file.write(line)
    os.system('chmod a+wr {}'.format(out_file_list_path))


def file_list():
    image_base = r'/rootfs/media/kasim/Data1/data/VideoFaceImage'
    file_list_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/filter_video_list.txt'
    out_file_list_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/filter_file_list.txt'
    with open(out_file_list_path, 'w') as out_file:
        with open(file_list_path, 'r') as file:
            for line in file.readlines():
                line = line.strip()
                image_dir = os.path.join(image_base, line)
                image_dir = os.path.normpath(image_dir)
                for dir_name, sub_dirs, files in os.walk(image_dir):
                    dir_name = os.path.normpath(dir_name)
                    if image_dir != dir_name:
                        # print(file_path, dir_name)
                        continue
                    file_paths = []
                    for file_name in files:
                        file_ext = os.path.splitext(file_name)[1].lower()
                        if file_ext != '.jpg':
                            continue
                        file_paths.append(os.path.join(line, file_name))
                    file_paths.sort()
                    for file_path in file_paths:
                        out_file.write(file_path + '\n')
    os.system('chmod a+wr {}'.format(out_file_list_path))


def filter_attr_video2():
    ANGLE_SCORE = 0.2
    ANGLE_DIFF = 15

    MIN_PITCH1 = -18.0
    MAX_PITCH1 = 18.0
    MIN_YAW1 = -18.0
    MAX_YAW1 = 18.0
    MIN_ROLL1 = -20.0
    MAX_ROLL1 = 20.0

    MIN_PITCH2 = -20.0
    MAX_PITCH2 = 20.0
    MIN_YAW2 = -20.0
    MAX_YAW2 = 20.0
    MIN_ROLL2 = -30.0
    MAX_ROLL2 = 30.0

    TOPK = 10

    face_attr_list_file = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/extract_image_list_72.attr.txt'
    face_rec_video_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/_filter_SelectFaces.txt'
    face_rec_images_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/BoxFeature.txt'

    images_set = set()
    with open(face_rec_images_path, 'r') as file:
        for line in file.readlines():
            images_set.add(line.split()[0])

    image_faceid_dict = {}
    with open(face_rec_video_path, 'r') as file:
        for line in file.readlines():
            # 1 gd/zs/sqqjd/dxssjysq/150100414a54443452064d2743233600/video/20190927_103958 000000_000.jpg,000001_000.jpg,000002_000.jpg,000003_000.jpg
            lines = line.strip().split()
            faceid = int(lines[0])
            video_name = lines[1]
            image_names = lines[2].split(',')
            for image_name in image_names:
                image_path = os.path.join(video_name, image_name)
                if image_path not in images_set:
                    continue
                image_faceid_dict[image_path] = faceid

    faceid_attrs = {}
    file_count = 0
    with open(face_attr_list_file, 'r') as file:
        isfirst = True
        for line in file.readlines():
            # file_name;bin_pitch;bin_roll;bin_yaw;pitch;roll;yaw;quality;mask
            # gd/zs/nqjd/dxjyjy/150100414a54443452064d2742c93600/video/20190817_030936/000000_000.jpg;-2.651,0.516;-2.601,0.566;-89.733,0.318;-1.642;-5.795;-31.306;0.045,0.150,0.421,0.384;72.692,73.769,63.882,65.441,77.657,95.700,72.600,64.765,65.577
            lines = line.strip().split(';')
            if isfirst:
                isfirst = False
                continue
            file_name = lines[0]
            if file_name not in images_set:
                continue
            bin_pitch = lines[1].split(',')
            bin_roll = lines[2].split(',')
            bin_yaw = lines[3].split(',')
            bin_pitch_score = float(bin_pitch[1])
            bin_roll_score = float(bin_roll[1])
            bin_yaw_score = float(bin_yaw[1])
            bin_pitch = float(bin_pitch[0])
            bin_roll = float(bin_roll[0])
            bin_yaw = float(bin_yaw[0])

            pitch = float(lines[4])
            roll = float(lines[5])
            yaw = float(lines[6])

            quality = lines[7].split(',')
            quality = 0.3333333333 * float(quality[2]) + 0.66666666 * float(quality[1]) + float(quality[0])

            faceid = image_faceid_dict[file_name]

            image_attrs = faceid_attrs.get(faceid, None)
            if image_attrs is None:
                faceid_attrs[faceid] = {'file_name': [file_name], 'pitch': [pitch], 'roll': [roll], 'yaw': [yaw], 'bin_pitch': [bin_pitch], 'bin_roll': [bin_roll], 'bin_yaw': [bin_yaw], 'bin_pitch_score': [bin_pitch_score], 'bin_roll_score': [bin_roll_score], 'bin_yaw_score': [bin_yaw_score], 'quality': [quality]}
            else:
                image_attrs['file_name'].append(file_name)
                image_attrs['pitch'].append(pitch)
                image_attrs['roll'].append(roll)
                image_attrs['yaw'].append(yaw)
                image_attrs['bin_pitch'].append(bin_pitch)
                image_attrs['bin_roll'].append(bin_roll)
                image_attrs['bin_yaw'].append(bin_yaw)
                image_attrs['bin_pitch_score'].append(bin_pitch_score)
                image_attrs['bin_roll_score'].append(bin_roll_score)
                image_attrs['bin_yaw_score'].append(bin_yaw_score)
                image_attrs['quality'].append(quality)
            file_count += 1
            # if file_count > 1000:
            #     break

    out_path = os.path.join(os.path.dirname(face_rec_video_path), 'faceid_typical_images.txt')
    with open(out_path, 'w') as file:
        faceid_typical_images = {}
        for faceid, image_attrs in faceid_attrs.items():
            file_names = image_attrs['file_name']
            pitch = np.array(image_attrs['pitch'], dtype=np.float32)
            roll = np.array(image_attrs['roll'], dtype=np.float32)
            yaw = np.array(image_attrs['yaw'], dtype=np.float32)
            bin_pitch = np.array(image_attrs['bin_pitch'], dtype=np.float32)
            bin_roll = np.array(image_attrs['bin_roll'], dtype=np.float32)
            bin_yaw = np.array(image_attrs['bin_yaw'], dtype=np.float32)
            bin_pitch_score = np.array(image_attrs['bin_pitch_score'], dtype=np.float32)
            bin_roll_score = np.array(image_attrs['bin_roll_score'], dtype=np.float32)
            bin_yaw_score = np.array(image_attrs['bin_yaw_score'], dtype=np.float32)
            quality = np.array(image_attrs['quality'], dtype=np.float32)

            pitch_diff = np.abs(bin_pitch - pitch)
            roll_diff = np.abs(bin_roll - roll)
            yaw_diff = np.abs(bin_yaw - yaw)

            score_mask = (bin_pitch_score > ANGLE_SCORE) & (bin_roll_score > ANGLE_SCORE) & (bin_yaw_score > ANGLE_SCORE)
            diff_mask = (pitch_diff < ANGLE_DIFF) & (roll_diff < ANGLE_DIFF) & (yaw_diff < ANGLE_DIFF)
            mask = score_mask & diff_mask
            if mask.any():
                pitch = pitch[mask]
                roll = roll[mask]
                yaw = yaw[mask]
                quality = quality[mask]
                file_names = [f for m, f in zip(mask, file_names) if m]
            idxes = np.arange(0, len(file_names))
            pitch_mask = np.logical_and(MIN_PITCH1 <= pitch, pitch <= MAX_PITCH1)
            roll_mask = np.logical_and(MIN_ROLL1 <= roll, roll <= MAX_ROLL1)
            yaw_mask = np.logical_and(MIN_YAW1 <= yaw, yaw <= MAX_YAW1)
            mask = pitch_mask & roll_mask & yaw_mask
            if mask.any():
                max_quality_idx = quality[mask].argsort()
                idx = idxes[mask][max_quality_idx]
            else:
                pitch_mask = np.logical_and(MIN_PITCH2 <= pitch, pitch <= MAX_PITCH2)
                roll_mask = np.logical_and(MIN_ROLL2 <= roll, roll <= MAX_ROLL2)
                yaw_mask = np.logical_and(MIN_YAW2 <= yaw, yaw <= MAX_YAW2)
                mask = pitch_mask & roll_mask & yaw_mask
                if mask.any():
                    max_quality_idx = quality[mask].argsort()
                    idx = idxes[mask][max_quality_idx]
                else:
                    # idx = quality.argsort()[0:1]
                    idx = quality.argsort()
            if idx.shape[0] > TOPK:
                idx = idx[0:TOPK]
            file_names = [file_names[_idx] for _idx in idx]
            faceid_typical_images[faceid] = file_names

        faceids = list(faceid_typical_images.keys())
        faceids.sort()
        for faceid in faceids:
            images = faceid_typical_images[faceid]
            images = list(images)
            images.sort()
            for image in images:
                out_info = '{} {}\n'.format(faceid, image)
                file.write(out_info)

    os.system('chmod a+wr {}'.format(out_path))

    print('Finish!')


def filter_attr_video3():
    ANGLE_SCORE = 0.2
    ANGLE_DIFF = 15

    MIN_PITCH1 = -18.0
    MAX_PITCH1 = 18.0
    MIN_YAW1 = -18.0
    MAX_YAW1 = 18.0
    MIN_ROLL1 = -20.0
    MAX_ROLL1 = 20.0

    MIN_PITCH2 = -20.0
    MAX_PITCH2 = 20.0
    MIN_YAW2 = -20.0
    MAX_YAW2 = 20.0
    MIN_ROLL2 = -30.0
    MAX_ROLL2 = 30.0

    MIN_PITCH3 = -30.0
    MAX_PITCH3 = 30.0
    MIN_YAW3 = -30.0
    MAX_YAW3 = 30.0
    MIN_ROLL3 = -45.0
    MAX_ROLL3 = 45.0

    TOPK = 10
    IMAGE_TOPK = 4

    face_attr_list_file = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/extract_image_list_72.attr.txt'
    face_rec_video_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/_filter_SelectFaces.txt'
    face_rec_images_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/BoxFeature.txt'

    random.seed(12345)

    images_set = set()
    with open(face_rec_images_path, 'r') as file:
        for line in file.readlines():
            images_set.add(line.split()[0])

    faceid_videos_dict = {}
    image_faceid_dict = {}
    with open(face_rec_video_path, 'r') as file:
        for line in file.readlines():
            # 1 gd/zs/sqqjd/dxssjysq/150100414a54443452064d2743233600/video/20190927_103958 000000_000.jpg,000001_000.jpg,000002_000.jpg,000003_000.jpg
            lines = line.strip().split()
            faceid = int(lines[0])
            video_name = lines[1]
            video_paths = faceid_videos_dict.get(faceid, None)
            if video_paths is None:
                faceid_videos_dict[faceid] = {video_name}
            else:
                video_paths.add(video_name)
            image_names = lines[2].split(',')
            for image_name in image_names:
                image_path = os.path.join(video_name, image_name)
                if image_path not in images_set:
                    continue
                image_faceid_dict[image_path] = faceid

    video_image_attrs = {}
    file_count = 0
    with open(face_attr_list_file, 'r') as file:
        isfirst = True
        for line in file.readlines():
            # file_name;bin_pitch;bin_roll;bin_yaw;pitch;roll;yaw;quality;mask
            # gd/zs/nqjd/dxjyjy/150100414a54443452064d2742c93600/video/20190817_030936/000000_000.jpg;-2.651,0.516;-2.601,0.566;-89.733,0.318;-1.642;-5.795;-31.306;0.045,0.150,0.421,0.384;72.692,73.769,63.882,65.441,77.657,95.700,72.600,64.765,65.577
            lines = line.strip().split(';')
            if isfirst:
                isfirst = False
                continue
            file_name = lines[0]
            if file_name not in images_set:
                continue
            bin_pitch = lines[1].split(',')
            bin_roll = lines[2].split(',')
            bin_yaw = lines[3].split(',')
            bin_pitch_score = float(bin_pitch[1])
            bin_roll_score = float(bin_roll[1])
            bin_yaw_score = float(bin_yaw[1])
            bin_pitch = float(bin_pitch[0])
            bin_roll = float(bin_roll[0])
            bin_yaw = float(bin_yaw[0])

            pitch = float(lines[4])
            roll = float(lines[5])
            yaw = float(lines[6])

            quality = lines[7].split(',')
            quality = 0.3333333333 * float(quality[2]) + 0.66666666 * float(quality[1]) + float(quality[0])
            video_name = os.path.dirname(file_name)
            image_attrs = video_image_attrs.get(video_name, None)
            if image_attrs is None:
                video_image_attrs[video_name] = {'file_name': [file_name], 'pitch': [pitch], 'roll': [roll], 'yaw': [yaw], 'bin_pitch': [bin_pitch], 'bin_roll': [bin_roll], 'bin_yaw': [bin_yaw], 'bin_pitch_score': [bin_pitch_score], 'bin_roll_score': [bin_roll_score], 'bin_yaw_score': [bin_yaw_score], 'quality': [quality]}
            else:
                image_attrs['file_name'].append(file_name)
                image_attrs['pitch'].append(pitch)
                image_attrs['roll'].append(roll)
                image_attrs['yaw'].append(yaw)
                image_attrs['bin_pitch'].append(bin_pitch)
                image_attrs['bin_roll'].append(bin_roll)
                image_attrs['bin_yaw'].append(bin_yaw)
                image_attrs['bin_pitch_score'].append(bin_pitch_score)
                image_attrs['bin_roll_score'].append(bin_roll_score)
                image_attrs['bin_yaw_score'].append(bin_yaw_score)
                image_attrs['quality'].append(quality)
            file_count += 1
            # if file_count > 1000:
            #     break

    video_typical_image_dict = {}
    for video_name, image_attrs in video_image_attrs.items():
        file_names = image_attrs['file_name']
        pitch = np.array(image_attrs['pitch'], dtype=np.float32)
        roll = np.array(image_attrs['roll'], dtype=np.float32)
        yaw = np.array(image_attrs['yaw'], dtype=np.float32)
        bin_pitch = np.array(image_attrs['bin_pitch'], dtype=np.float32)
        bin_roll = np.array(image_attrs['bin_roll'], dtype=np.float32)
        bin_yaw = np.array(image_attrs['bin_yaw'], dtype=np.float32)
        bin_pitch_score = np.array(image_attrs['bin_pitch_score'], dtype=np.float32)
        bin_roll_score = np.array(image_attrs['bin_roll_score'], dtype=np.float32)
        bin_yaw_score = np.array(image_attrs['bin_yaw_score'], dtype=np.float32)
        quality = np.array(image_attrs['quality'], dtype=np.float32)

        pitch_diff = np.abs(bin_pitch - pitch)
        roll_diff = np.abs(bin_roll - roll)
        yaw_diff = np.abs(bin_yaw - yaw)

        score_mask = (bin_pitch_score > ANGLE_SCORE) & (bin_roll_score > ANGLE_SCORE) & (bin_yaw_score > ANGLE_SCORE)
        diff_mask = (pitch_diff < ANGLE_DIFF) & (roll_diff < ANGLE_DIFF) & (yaw_diff < ANGLE_DIFF)
        mask = score_mask & diff_mask
        if mask.any():
            pitch = pitch[mask]
            roll = roll[mask]
            yaw = yaw[mask]
            quality = quality[mask]
            file_names = [f for m, f in zip(mask, file_names) if m]
        idxes = np.arange(0, len(file_names))
        pitch_mask = np.logical_and(MIN_PITCH1 <= pitch, pitch <= MAX_PITCH1)
        roll_mask = np.logical_and(MIN_ROLL1 <= roll, roll <= MAX_ROLL1)
        yaw_mask = np.logical_and(MIN_YAW1 <= yaw, yaw <= MAX_YAW1)
        mask = pitch_mask & roll_mask & yaw_mask
        if mask.any():
            max_quality_idx = quality[mask].argsort()
            idx = idxes[mask][max_quality_idx]
        else:
            pitch_mask = np.logical_and(MIN_PITCH2 <= pitch, pitch <= MAX_PITCH2)
            roll_mask = np.logical_and(MIN_ROLL2 <= roll, roll <= MAX_ROLL2)
            yaw_mask = np.logical_and(MIN_YAW2 <= yaw, yaw <= MAX_YAW2)
            mask = pitch_mask & roll_mask & yaw_mask
            if mask.any():
                max_quality_idx = quality[mask].argsort()
                idx = idxes[mask][max_quality_idx]
            else:
                pitch_mask = np.logical_and(MIN_PITCH3 <= pitch, pitch <= MAX_PITCH3)
                roll_mask = np.logical_and(MIN_ROLL3 <= roll, roll <= MAX_ROLL3)
                yaw_mask = np.logical_and(MIN_YAW3 <= yaw, yaw <= MAX_YAW3)
                mask = pitch_mask & roll_mask & yaw_mask
                if mask.any():
                    max_quality_idx = quality[mask].argsort()
                    idx = idxes[mask][max_quality_idx]
                else:
                    idx = quality.argsort()[0:2]
                    # idx = quality.argsort()
        if idx.shape[0] > IMAGE_TOPK:
            idx = idx[0:IMAGE_TOPK]
        file_names = [file_names[_idx] for _idx in idx]
        video_typical_image_dict[video_name] = file_names

    out_path = os.path.join(os.path.dirname(face_rec_video_path), 'faceid_typical_images.txt')
    with open(out_path, 'w') as file:
        faceid_typical_images = {}
        faceids = list(faceid_videos_dict.keys())
        for faceid in faceids:
            videos = faceid_videos_dict[faceid]
            typical_images = []
            left_typical_images = []
            for video in videos:
                images = video_typical_image_dict.get(video, None)
                if (images is None) or (len(images) < 1):
                    continue
                typical_images.append(images[0])
                if len(images) > 1:
                    left_typical_images.extend(images[1:])
            typical_image_count = len(typical_images)
            if TOPK > typical_image_count:
                select_typical_image_count = TOPK - typical_image_count
                random.shuffle(left_typical_images)
                select_typical_images = left_typical_images[:select_typical_image_count]
                typical_images.extend(select_typical_images)
            if len(left_typical_images) > 0:
                faceid_typical_images[faceid] = typical_images

        faceids = list(faceid_typical_images.keys())
        faceids.sort()
        for faceid in faceids:
            images = faceid_typical_images[faceid]
            images = list(images)
            images.sort()
            for image in images:
                out_info = '{} {}\n'.format(faceid, image)
                file.write(out_info)

    os.system('chmod a+wr {}'.format(out_path))

    print('Finish!')


def filter_attr_video4():
    ANGLE_SCORE = 0.2
    ANGLE_DIFF = 15

    MIN_PITCH1 = -18.0
    MAX_PITCH1 = 18.0
    MIN_YAW1 = -18.0
    MAX_YAW1 = 18.0
    MIN_ROLL1 = -20.0
    MAX_ROLL1 = 20.0

    MIN_PITCH2 = -20.0
    MAX_PITCH2 = 20.0
    MIN_YAW2 = -20.0
    MAX_YAW2 = 20.0
    MIN_ROLL2 = -30.0
    MAX_ROLL2 = 30.0

    MIN_PITCH3 = -30.0
    MAX_PITCH3 = 30.0
    MIN_YAW3 = -30.0
    MAX_YAW3 = 30.0
    MIN_ROLL3 = -45.0
    MAX_ROLL3 = 45.0

    TOPK = 10

    face_attr_list_file = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/extract_image_list_72.attr.txt'
    face_rec_video_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/RFaceRec.txt'

    image_faceid_dict = {}
    with open(face_rec_video_path, 'r') as file:
        for line in file.readlines():
            # 1 gd/zs/sqqjd/dxssjysq/150100414a54443452064d2c43ed3600/video/20190922_201627/000024_000.jpg
            lines = line.strip().split()
            faceid = int(lines[0])
            image_path = lines[1]
            image_faceid_dict[image_path] = faceid

    images_set = image_faceid_dict.keys()
    faceid_attrs = {}
    file_count = 0
    with open(face_attr_list_file, 'r') as file:
        isfirst = True
        for line in file.readlines():
            # file_name;bin_pitch;bin_roll;bin_yaw;pitch;roll;yaw;quality;mask
            # gd/zs/nqjd/dxjyjy/150100414a54443452064d2742c93600/video/20190817_030936/000000_000.jpg;-2.651,0.516;-2.601,0.566;-89.733,0.318;-1.642;-5.795;-31.306;0.045,0.150,0.421,0.384;72.692,73.769,63.882,65.441,77.657,95.700,72.600,64.765,65.577
            lines = line.strip().split(';')
            if isfirst:
                isfirst = False
                continue
            file_name = lines[0]
            if file_name not in images_set:
                continue
            bin_pitch = lines[1].split(',')
            bin_roll = lines[2].split(',')
            bin_yaw = lines[3].split(',')
            bin_pitch_score = float(bin_pitch[1])
            bin_roll_score = float(bin_roll[1])
            bin_yaw_score = float(bin_yaw[1])
            bin_pitch = float(bin_pitch[0])
            bin_roll = float(bin_roll[0])
            bin_yaw = float(bin_yaw[0])

            pitch = float(lines[4])
            roll = float(lines[5])
            yaw = float(lines[6])

            quality = lines[7].split(',')
            quality = 0.3333333333 * float(quality[2]) + 0.66666666 * float(quality[1]) + float(quality[0])

            faceid = image_faceid_dict[file_name]

            image_attrs = faceid_attrs.get(faceid, None)
            if image_attrs is None:
                faceid_attrs[faceid] = {'file_name': [file_name], 'pitch': [pitch], 'roll': [roll], 'yaw': [yaw], 'bin_pitch': [bin_pitch], 'bin_roll': [bin_roll], 'bin_yaw': [bin_yaw], 'bin_pitch_score': [bin_pitch_score], 'bin_roll_score': [bin_roll_score], 'bin_yaw_score': [bin_yaw_score], 'quality': [quality]}
            else:
                image_attrs['file_name'].append(file_name)
                image_attrs['pitch'].append(pitch)
                image_attrs['roll'].append(roll)
                image_attrs['yaw'].append(yaw)
                image_attrs['bin_pitch'].append(bin_pitch)
                image_attrs['bin_roll'].append(bin_roll)
                image_attrs['bin_yaw'].append(bin_yaw)
                image_attrs['bin_pitch_score'].append(bin_pitch_score)
                image_attrs['bin_roll_score'].append(bin_roll_score)
                image_attrs['bin_yaw_score'].append(bin_yaw_score)
                image_attrs['quality'].append(quality)
            file_count += 1
            # if file_count > 1000:
            #     break

    out_path = os.path.join(os.path.dirname(face_rec_video_path), 'faceid_typical_images.txt')
    with open(out_path, 'w') as file:
        faceid_typical_images = {}
        for faceid, image_attrs in faceid_attrs.items():
            file_names = image_attrs['file_name']
            pitch = np.array(image_attrs['pitch'], dtype=np.float32)
            roll = np.array(image_attrs['roll'], dtype=np.float32)
            yaw = np.array(image_attrs['yaw'], dtype=np.float32)
            bin_pitch = np.array(image_attrs['bin_pitch'], dtype=np.float32)
            bin_roll = np.array(image_attrs['bin_roll'], dtype=np.float32)
            bin_yaw = np.array(image_attrs['bin_yaw'], dtype=np.float32)
            bin_pitch_score = np.array(image_attrs['bin_pitch_score'], dtype=np.float32)
            bin_roll_score = np.array(image_attrs['bin_roll_score'], dtype=np.float32)
            bin_yaw_score = np.array(image_attrs['bin_yaw_score'], dtype=np.float32)
            quality = np.array(image_attrs['quality'], dtype=np.float32)

            pitch_diff = np.abs(bin_pitch - pitch)
            roll_diff = np.abs(bin_roll - roll)
            yaw_diff = np.abs(bin_yaw - yaw)

            score_mask = (bin_pitch_score > ANGLE_SCORE) & (bin_roll_score > ANGLE_SCORE) & (bin_yaw_score > ANGLE_SCORE)
            diff_mask = (pitch_diff < ANGLE_DIFF) & (roll_diff < ANGLE_DIFF) & (yaw_diff < ANGLE_DIFF)
            mask = score_mask & diff_mask
            if mask.any():
                pitch = pitch[mask]
                roll = roll[mask]
                yaw = yaw[mask]
                quality = quality[mask]
                file_names = [f for m, f in zip(mask, file_names) if m]
            idxes = np.arange(0, len(file_names))
            pitch_mask = np.logical_and(MIN_PITCH1 <= pitch, pitch <= MAX_PITCH1)
            roll_mask = np.logical_and(MIN_ROLL1 <= roll, roll <= MAX_ROLL1)
            yaw_mask = np.logical_and(MIN_YAW1 <= yaw, yaw <= MAX_YAW1)
            mask = pitch_mask & roll_mask & yaw_mask
            if mask.any():
                max_quality_idx = quality[mask].argsort()
                idx = idxes[mask][max_quality_idx]
            else:
                pitch_mask = np.logical_and(MIN_PITCH2 <= pitch, pitch <= MAX_PITCH2)
                roll_mask = np.logical_and(MIN_ROLL2 <= roll, roll <= MAX_ROLL2)
                yaw_mask = np.logical_and(MIN_YAW2 <= yaw, yaw <= MAX_YAW2)
                mask = pitch_mask & roll_mask & yaw_mask
                if mask.any():
                    max_quality_idx = quality[mask].argsort()
                    idx = idxes[mask][max_quality_idx]
                else:
                    pitch_mask = np.logical_and(MIN_PITCH3 <= pitch, pitch <= MAX_PITCH3)
                    roll_mask = np.logical_and(MIN_ROLL3 <= roll, roll <= MAX_ROLL3)
                    yaw_mask = np.logical_and(MIN_YAW3 <= yaw, yaw <= MAX_YAW3)
                    mask = pitch_mask & roll_mask & yaw_mask
                    if mask.any():
                        max_quality_idx = quality[mask].argsort()
                        idx = idxes[mask][max_quality_idx]
                    else:
                        # idx = quality.argsort()[0:1]
                        idx = quality.argsort()
            if idx.shape[0] > TOPK:
                idx = idx[0:TOPK]
            file_names = [file_names[_idx] for _idx in idx]
            faceid_typical_images[faceid] = file_names

        faceids = list(faceid_typical_images.keys())
        faceids.sort()
        for faceid in faceids:
            images = faceid_typical_images[faceid]
            images = list(images)
            images.sort()
            for image in images:
                out_info = '{} {}\n'.format(faceid, image)
                file.write(out_info)

    os.system('chmod a+wr {}'.format(out_path))

    print('Finish!')


def read_features(file_list, feature_file, feat_size=1032, feat_head_offset=8):
    file_features = {}
    feat_file = open(feature_file, 'rb')
    file_count = 0
    with open(file_list, 'r') as file:
        for line in file.readlines():
            file_count += 1
            if file_count % 10000 == 0:
                print('Read Feature Count: {}'.format(file_count))

            # if file_count == 100000:
            #     break
            # gd/zs/sqqjd/dxssjysq/150100414a54443452064d2743233600/video/20190927_103958/000000_000.jpg -13,44,240,298,1,1032
            lines = line.split()
            file_name = lines[0]

            feature = feat_file.read(feat_size)
            feature = feature[feat_head_offset:]

            file_features[file_name] = feature
        print('Read Feature Count: {}'.format(file_count))
    feat_file.close()

    return file_features


def write_features(out_file_list_path, out_feature_file_path, image_faceids, image_features):
    faceid_image_features = {}
    for image, faceid in image_faceids.items():
        feature = image_features.get(image, None)
        if feature is None:
            continue
        _image_features = faceid_image_features.get(faceid, None)
        if _image_features is None:
            faceid_image_features[faceid] = [[image, feature]]
        else:
            _image_features.append([image, feature])

    faceids = list(faceid_image_features.keys())
    faceids.sort()
    out_file = open(out_file_list_path, 'w')
    out_feature_file = open(out_feature_file_path, 'wb')
    for faceid in faceids:
        _image_features = faceid_image_features[faceid]
        _image_features.sort(key=lambda x: x[0])
        for image, features in _image_features:
            out_info = '{} {}\n'.format(faceid, image)
            out_file.write(out_info)
            out_feature_file.write(features)
    out_file.close()
    out_feature_file.close()
    os.system('chmod a+wr {}'.format(out_file_list_path))
    os.system('chmod a+wr {}'.format(out_feature_file_path))


def filter_features():
    file_list_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/BoxFeature.txt'
    feature_file_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/Feature.dat'
    face_rec_images_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/_filter_SelectFaces.txt'
    face_rec_typical_images_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/faceid_typical_images.txt'

    image_features = read_features(file_list_path, feature_file_path)

    faceid_set = set()
    typical_image_faceid_dict = {}
    with open(face_rec_typical_images_path, 'r') as file:
        for line in file.readlines():
            # 1 gd/zs/sqqjd/dxssjysq/150100414a54443452064d2743233600/video/20190927_103958/000000_000.jpg
            lines = line.strip().split()
            faceid = int(lines[0])
            image_path = lines[1]
            faceid_set.add(faceid)
            typical_image_faceid_dict[image_path] = faceid

    image_faceid_dict = {}
    with open(face_rec_images_path, 'r') as file:
        for line in file.readlines():
            # 1 gd/zs/sqqjd/dxssjysq/150100414a54443452064d2743233600/video/20190927_103958 000000_000.jpg,000001_000.jpg,000002_000.jpg,000003_000.jpg
            lines = line.strip().split()
            faceid = int(lines[0])
            if faceid not in faceid_set:
                continue
            video_name = lines[1]
            image_names = lines[2].split(',')
            for image_name in image_names:
                image_path = os.path.join(video_name, image_name)
                image_faceid_dict[image_path] = faceid

    out_file_list_path = '/rootfs/media/kasim/Data1/data/VideoFaceImage/TypicalFaceRec.txt'
    out_feature_file_path = '/rootfs/media/kasim/Data1/data/VideoFaceImage/TypicalFaceRecFeat.dat'
    write_features(out_file_list_path, out_feature_file_path, typical_image_faceid_dict, image_features)

    out_file_list_path = '/rootfs/media/kasim/Data1/data/VideoFaceImage/FaceRec.txt'
    out_feature_file_path = '/rootfs/media/kasim/Data1/data/VideoFaceImage/FaceRecFeat.dat'
    write_features(out_file_list_path, out_feature_file_path, image_faceid_dict, image_features)


def filter_features2():
    file_list_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/BoxFeature.txt'
    feature_file_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/Feature.dat'
    face_rec_typical_images_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/faceid_typical_images.txt'

    image_features = read_features(file_list_path, feature_file_path)

    faceid_set = set()
    typical_image_faceid_dict = {}
    with open(face_rec_typical_images_path, 'r') as file:
        for line in file.readlines():
            # 1 gd/zs/sqqjd/dxssjysq/150100414a54443452064d2743233600/video/20190927_103958/000000_000.jpg
            lines = line.strip().split()
            faceid = int(lines[0])
            image_path = lines[1]
            faceid_set.add(faceid)
            typical_image_faceid_dict[image_path] = faceid

    out_file_list_path = '/rootfs/media/kasim/Data1/data/VideoFaceImage/TypicalFaceRec.txt'
    out_feature_file_path = '/rootfs/media/kasim/Data1/data/VideoFaceImage/TypicalFaceRecFeat.dat'
    write_features(out_file_list_path, out_feature_file_path, typical_image_faceid_dict, image_features)


def filter_faceid_video():
    # file_list_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/RTypicalFaceRec.txt'
    file_list_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/RFaceRec.txt'
    faceid_videos = {}
    with open(file_list_path, 'r') as file:
        for line in file.readlines():
            # 1 gd/zs/sqqjd/dxssjysq/150100414a54443452064d2743233600/video/20190927_103958/000056_000.jpg
            lines = line.strip().split()
            faceid = int(lines[0])
            video = os.path.dirname(lines[1])
            videos = faceid_videos.get(faceid, None)
            if videos is None:
                faceid_videos[faceid] = {video}
            else:
                videos.add(video)

    out_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/FaceIdVideos.txt'
    face_count = 0
    file_count = 0
    with open(out_path, 'w') as file:
        faceids = list(faceid_videos.keys())
        faceids.sort()
        for faceid in faceids:
            videos = faceid_videos[faceid]
            if len(videos) < 1:
                continue
            face_count += 1
            videos = list(videos)
            videos.sort()
            for video in videos:
                out_info = '{} {}\n'.format(faceid, video)
                file.write(out_info)
                file_count += 1
    os.system('chmod a+wr {}'.format(out_path))
    print('Face Count:', face_count, ', Face Video Count:', file_count)

    out_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/FaceIdVideos2.txt'
    face_count = 0
    file_count = 0
    with open(out_path, 'w') as file:
        faceids = list(faceid_videos.keys())
        faceids.sort()
        for faceid in faceids:
            videos = faceid_videos[faceid]
            if len(videos) < 1:
                continue
            videos = list(videos)
            videos.sort()
            if len(videos) > 1:
                face_count += 1
                for video in videos:
                    out_info = '{} {}\n'.format(faceid, video)
                    file.write(out_info)
                    file_count += 1
    os.system('chmod a+wr {}'.format(out_path))
    print('Face Count:', face_count, ', Face Video Count:', file_count)


def filter_image_bbox():
    face_rec_images_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/filter_SelectFaces.txt'
    image_bboxes_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/extract_image_list_72.txt'

    images_set = set()
    with open(face_rec_images_path, 'r') as file:
        for line in file.readlines():
            images_set.add(line.strip())

    out_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/image_bboxes.txt'
    out_file = open(out_path, 'w')
    with open(image_bboxes_path, 'r') as file:
        for line in file.readlines():
            image = line.split()[0]
            if image not in images_set:
                continue
            out_file.write(line)
    out_file.close()
    os.system('chmod a+wr {}'.format(out_path))


def split_face_rec_video2():
    MAX_VIDEO_COUNT = 10
    typical_face_rec_video_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/_FaceIdVideos2.txt'
    face_rec_video_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/FaceIdVideos2.txt'
    random.seed(123456)

    faceid_video_path_dict = {}
    with open(face_rec_video_path, 'r') as file:
        for line in file.readlines():
            # 1 gd/zs/sqqjd/dxssjysq/150100414a54443452064d2743233600/video/20190927_103958
            lines = line.strip().split()
            faceid = int(lines[0])
            video_path = lines[1]
            video_paths = faceid_video_path_dict.get(faceid, None)
            if video_paths is None:
                faceid_video_path_dict[faceid] = {video_path}
            else:
                video_paths.add(video_path)

    typical_faceid_video_path_dict = {}
    with open(typical_face_rec_video_path, 'r') as file:
        for line in file.readlines():
            # 1 gd/zs/sqqjd/dxssjysq/150100414a54443452064d2743233600/video/20190927_103958
            lines = line.strip().split()
            faceid = int(lines[0])
            video_path = lines[1]
            video_paths = typical_faceid_video_path_dict.get(faceid, None)
            if video_paths is None:
                typical_faceid_video_path_dict[faceid] = {video_path}
            else:
                video_paths.add(video_path)

    faceids = list(typical_faceid_video_path_dict.keys())
    faceids.sort()
    faceid_count = 0
    gallery_file_count = 0
    query_file_count = 0
    query_file_path = os.path.join(os.path.dirname(face_rec_video_path), 'face_rec_query_video_list.txt')
    gallery_file_path = os.path.join(os.path.dirname(face_rec_video_path), 'face_rec_gallery_video_list.txt')
    query_file = open(query_file_path, 'w')
    gallery_file = open(gallery_file_path, 'w')
    for faceid in faceids:
        video_paths = typical_faceid_video_path_dict[faceid]
        video_count = len(video_paths)
        if video_count < 1:
            continue
        _video_paths = list(video_paths)
        random.shuffle(_video_paths)

        gallery_count = video_count // 2
        if gallery_count < 1:
            gallery_count = 1
        gallery_paths = _video_paths[0:gallery_count]
        query_paths = _video_paths[gallery_count:video_count]
        query_count = len(query_paths)

        all_video_paths = faceid_video_path_dict.get(faceid, None)
        if all_video_paths is not None:
            remain_video_paths = [v for v in all_video_paths if v not in video_paths]
            remain_video_count = len(remain_video_paths)
            if remain_video_count > 0:
                random.shuffle(remain_video_paths)
                remain_gallery_count = remain_video_count // 2
                remain_gallery_paths = remain_video_paths[0:remain_gallery_count]
                remain_query_paths = remain_video_paths[remain_gallery_count:remain_video_count]
                remain_query_count = len(remain_query_paths)
                if remain_gallery_count > 0:
                    gallery_paths.extend(remain_gallery_paths)
                    gallery_count += remain_gallery_count
                if remain_query_count > 0:
                    query_paths.extend(remain_query_paths)
                    query_count += remain_query_count

        if gallery_count > MAX_VIDEO_COUNT:
            gallery_paths = gallery_paths[0:MAX_VIDEO_COUNT]
        if query_count > MAX_VIDEO_COUNT:
            query_paths = query_paths[0:MAX_VIDEO_COUNT]

        _query_file_count = 0
        _gallery_file_count = 0
        faceid_count += 1
        for query_path in query_paths:
            out_info = '{} {}\n'.format(faceid, query_path)
            query_file.write(out_info)
            _query_file_count += 1
        for gallery_path in gallery_paths:
            out_info = '{} {}\n'.format(faceid, gallery_path)
            gallery_file.write(out_info)
            _gallery_file_count += 1

        gallery_file_count += _gallery_file_count
        query_file_count += _query_file_count

    query_file.close()
    gallery_file.close()
    os.system('chmod a+wr {}'.format(query_file_path))
    os.system('chmod a+wr {}'.format(gallery_file_path))
    print('Face Count:', faceid_count, ', Face Query Video Count:', query_file_count, ', Face Gallery Video Count:', gallery_file_count, ', Face Video Count:', (query_file_count+gallery_file_count))

    print('Finish!')


def split_face_rec_feature():
    face_rec_query_video_list_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/face_rec_query_video_list.txt'
    face_rec_gallery_video_list_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/face_rec_gallery_video_list.txt'

    file_list_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/BoxFeature.txt'
    feature_file_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/Feature.dat'
    face_rec_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/RFaceRec.txt'

    image_features = read_features(file_list_path, feature_file_path)

    def read_faceid_videos(video_list_path):
        query_faceid_videos = {}
        with open(video_list_path, 'r') as file:
            for line in file.readlines():
                # 1 gd/zs/sqqjd/dxssjysq/150100414a54443452064d2743233600/video/20190927_103958
                lines = line.strip().split()
                faceid = int(lines[0])
                video_path = lines[1]
                videos = query_faceid_videos.get(faceid, None)
                if videos is None:
                    query_faceid_videos[faceid] = {video_path}
                else:
                    videos.add(video_path)
        return query_faceid_videos

    query_faceid_videos = read_faceid_videos(face_rec_query_video_list_path)
    gallery_faceid_videos = read_faceid_videos(face_rec_gallery_video_list_path)

    query_faceid_video_images = {}
    gallery_faceid_video_images = {}

    query_image_faceid_dict = {}
    gallery_image_faceid_dict = {}
    with open(face_rec_path, 'r') as file:
        for line in file.readlines():
            # 1 gd/zs/sqqjd/dxssjysq/150100414a54443452064d2743233600/video/20190927_103958/000000_000.jpg
            lines = line.strip().split()
            faceid = int(lines[0])
            image_path = lines[1]
            video_path = os.path.dirname(image_path)

            image_base = os.path.basename(image_path)
            videos = query_faceid_videos.get(faceid, None)
            if videos is not None:
                if video_path in videos:
                    query_image_faceid_dict[image_path] = faceid
                    video_images = query_faceid_video_images.get(faceid, None)
                    if video_images is None:
                        query_faceid_video_images[faceid] = {video_path: [image_base]}
                    else:
                        images = video_images.get(video_path, None)
                        if images is None:
                            video_images[video_path] = [image_base]
                        else:
                            images.append(image_base)

            videos = gallery_faceid_videos.get(faceid, None)
            if videos is not None:
                if video_path in videos:
                    gallery_image_faceid_dict[image_path] = faceid
                    video_images = gallery_faceid_video_images.get(faceid, None)
                    if video_images is None:
                        gallery_faceid_video_images[faceid] = {video_path: [image_base]}
                    else:
                        images = video_images.get(video_path, None)
                        if images is None:
                            video_images[video_path] = [image_base]
                        else:
                            images.append(image_base)

    out_query_file_list_path = '/rootfs/media/kasim/Data1/data/VideoFaceImage/QueryFaceRec.txt'
    out_feature_file_path = '/rootfs/media/kasim/Data1/data/VideoFaceImage/QueryFaceRecFeat.dat'
    write_features(out_query_file_list_path, out_feature_file_path, query_image_faceid_dict, image_features)

    out_gallery_file_list_path = '/rootfs/media/kasim/Data1/data/VideoFaceImage/GalleryFaceRec.txt'
    out_feature_file_path = '/rootfs/media/kasim/Data1/data/VideoFaceImage/GalleryFaceRecFeat.dat'
    write_features(out_gallery_file_list_path, out_feature_file_path, gallery_image_faceid_dict, image_features)

    def write_faceid_video_images(out_file_list_path, faceid_video_images):
        with open(out_file_list_path, 'w') as file:
            faceids = list(faceid_video_images.keys())
            faceids.sort()
            face_count = 0
            video_count = 0
            image_count = 0
            for faceid in faceids:
                face_count += 1
                video_images = faceid_video_images[faceid]
                videos = list(video_images.keys())
                videos.sort()
                for video in videos:
                    video_count += 1
                    images = video_images[video]
                    image_count += len(images)
                    images.sort()
                    images_str = ','.join(images)
                    out_info = '{} {} {}\n'.format(faceid, video, images_str)
                    file.write(out_info)
        os.system('chmod a+wr {}'.format(out_file_list_path))
        print('{} Face Count: {}, Video Count: {}, Image Count: {}\n'.format(out_file_list_path, face_count, video_count, image_count))

    out_file_list_path = '/rootfs/media/kasim/Data1/data/VideoFaceImage/QueryFaceIdVideoImages.txt'
    write_faceid_video_images(out_file_list_path, query_faceid_video_images)

    out_file_list_path = '/rootfs/media/kasim/Data1/data/VideoFaceImage/GalleryFaceIdVideoImages.txt'
    write_faceid_video_images(out_file_list_path, gallery_faceid_video_images)

    out_images_list_path = '/rootfs/media/kasim/Data1/data/VideoFaceImage/QueryImages.txt'
    with open(out_query_file_list_path, 'r') as file:
        with open(out_images_list_path, 'w') as out_file:
            for line in file.readlines():
                image_path = line.split(' ')[1]
                out_file.write(image_path)
    os.system('chmod a+wr {}'.format(out_images_list_path))

    out_images_list_path = '/rootfs/media/kasim/Data1/data/VideoFaceImage/GalleryImages.txt'
    with open(out_gallery_file_list_path, 'r') as file:
        with open(out_images_list_path, 'w') as out_file:
            for line in file.readlines():
                image_path = line.split(' ')[1]
                out_file.write(image_path)
    os.system('chmod a+wr {}'.format(out_images_list_path))


def gaze_video_statistics():
    GAZE_COUNT = 1
    GAZE_THR = 20
    ANGLE_SCORE = 0.1
    ANGLE_DIFF = 20
    face_attr_list_file = r'/rootfs/media/kasim/Data1/data/VideoFaceImage/extract_image_list_72.attr.txt'
    face_rec_video_path = r'/rootfs/media/kasim/DataSet/RecordEx/one_person_face_video_list.txt'

    video_path_gaze_dict = {}
    with open(face_rec_video_path, 'r') as file:
        for line in file.readlines():
            lines = line.strip().split()
            video_path = os.path.splitext(lines[0])[0]
            video_path_gaze_dict[video_path] = 0

    with open(face_attr_list_file, 'r') as file:
        isfirst = True
        for line in file.readlines():
            # file_name;bin_pitch;bin_roll;bin_yaw;pitch;roll;yaw;quality;mask
            # gd/zs/nqjd/dxjyjy/150100414a54443452064d2742c93600/video/20190817_030936/000000_000.jpg;-2.651,0.516;-2.601,0.566;-89.733,0.318;-1.642;-5.795;-31.306;0.045,0.150,0.421,0.384;72.692,73.769,63.882,65.441,77.657,95.700,72.600,64.765,65.577
            lines = line.strip().split(';')
            if isfirst:
                isfirst = False
                continue

            file_name = os.path.dirname(lines[0])
            gaze_count = video_path_gaze_dict.get(file_name, None)
            if gaze_count is None:
                continue

            pitch = float(lines[4])
            roll = float(lines[5])
            yaw = float(lines[6])

            bin_pitch = lines[1].split(',')
            bin_roll = lines[2].split(',')
            bin_yaw = lines[3].split(',')
            if (float(bin_pitch[1]) < ANGLE_SCORE) or (float(bin_roll[1]) < ANGLE_SCORE) or (float(bin_yaw[1]) < ANGLE_SCORE):
                continue

            if (abs(float(bin_pitch[0]) - pitch) > ANGLE_DIFF) or (abs(float(bin_roll[0]) - roll) > ANGLE_DIFF) or (abs(float(bin_yaw[0]) - yaw) > ANGLE_DIFF):
                continue

            gaze = estimate_gaze_from_headpose(yaw, pitch, roll, gaze_thr=GAZE_THR)
            if gaze:
                video_path_gaze_dict[file_name] = gaze_count + 1

    total_video_count = len(video_path_gaze_dict)
    gaze_video_count = 0
    for video_name, gaze_count in video_path_gaze_dict.items():
        if gaze_count >= GAZE_COUNT:
            gaze_video_count += 1

    if total_video_count == 0:
        gaze_video_ratio = 0
    else:
        gaze_video_ratio = 100*gaze_video_count/total_video_count
    print('Gaze Video Count: {}, Total Video Count: {}, Gaze Video Ratio {:.03f}, No Gaze Video Ratio {:.03f}'.format(gaze_video_count, total_video_count, gaze_video_ratio, (100-gaze_video_ratio)))

    print('Finish!')


def filter_file():
    filter_file_list_path = '/rootfs/media/kasim/Data1/data/VideoCropFace/face_image_q5-10.txt'
    file_list_path = '/rootfs/media/kasim/Data1/data/VideoCropFace/face_image_q5-10_attr_ok.txt'
    out_file_list_path = os.path.splitext(file_list_path)[0] + '_filter.txt'

    filter_file_set = set()
    with open(filter_file_list_path, 'r') as file:
        for line in file.readlines():
            file_name = line.split()[0].strip()
            filter_file_set.add(file_name)

    with open(out_file_list_path, 'w') as out_file:
        with open(file_list_path, 'r') as file:
            for line in file.readlines():
                file_name = line.split()[0].strip()
                if file_name in filter_file_set:
                    out_file.write(line)
    os.system('chmod a+wr {}'.format(out_file_list_path))


def move_face_file():
    image_file_lists = [
        r'/rootfs/media/kasim/Data1/data/VideoFaceImage/QueryImages.txt',
        r'/rootfs/media/kasim/Data1/data/VideoFaceImage/GalleryImages.txt'
    ]
    image_base_path = r'/rootfs/media/kasim/Data1/data/VideoFaceImage'
    out_image_path = r'/rootfs/media/kasim/Data1/data/VideoFaceRecImage'
    if not os.path.exists(out_image_path):
        os.makedirs(out_image_path)
        os.system('chmod a+wr {}'.format(out_image_path))
    for image_file_list in image_file_lists:
        with open(image_file_list, 'r') as file:
            for line in file.readlines():
                image_path = line.strip().split()[0]
                src_path = os.path.join(image_base_path, image_path)
                if not os.path.exists(src_path):
                     continue
                dst_path = os.path.join(out_image_path, image_path)
                dir_name = os.path.dirname(dst_path)
                if not os.path.exists(dir_name):
                    os.makedirs(dir_name)
                shutil.move(src_path, dst_path)


def gender_filter():
    gender_file_path = r'/rootfs/media/yery/Kaso/data/VideoGenderCropFace/gender_images.txt'
    # gender_label_file_path = r'/rootfs/media/yery/Kaso/data/VideoGenderCropFace/female/select_images.txt'
    # currect_index_file_path = '/rootfs/media/yery/Kaso/data/VideoGenderCropFace/female/cur_img_idx.txt'
    gender_label_file_path = r'/rootfs/media/yery/Kaso/data/VideoGenderCropFace/male/select_images.txt'
    currect_index_file_path = '/rootfs/media/yery/Kaso/data/VideoGenderCropFace/male/cur_img_idx.txt'

    faceid_image_genders = {}
    with open(gender_file_path, 'r') as file:
        for line in file.readlines():
            lines = line.strip().split()
            file_path = lines[0]
            faceid = os.path.dirname(file_path)
            file_name = os.path.basename(file_path)
            gender = lines[1]
            faceid_image_genders[faceid] = [file_name, gender]

    currect_index = 0
    if os.path.exists(currect_index_file_path):
        with open(currect_index_file_path, 'r') as file:
            currect_index = int(file.readline().strip())

    faceid_image_labels = []
    with open(gender_label_file_path, 'r') as file:
        for line in file.readlines():
            lines = line.strip().split()
            file_path = lines[0]
            faceid = os.path.dirname(file_path)
            file_name = os.path.basename(file_path)
            gender = lines[1]
            faceid_image_labels.append([faceid, file_name, gender])

    out_path = os.path.splitext(gender_label_file_path)[0] + '.filter.txt'
    with open(out_path, 'w') as file:
        for idx, faceid_image_label in enumerate(faceid_image_labels):
            faceid, image, label = faceid_image_label
            image_gender = faceid_image_genders.get(faceid, None)
            if image_gender is None:
                gender = label
            else:
                image, gender = image_gender
            if idx < currect_index:
                gender = label
            out_info = '{}/{} {}\n'.format(faceid, image, gender)
            file.write(out_info)


def filter_video_crop_face_bbox():
    file_path = r'/rootfs/media/kasim/Data1/data/VideoGenderCropFace/video_crop_face_list.txt'
    out_file_path = r'/rootfs/media/kasim/Data1/data/VideoGenderCropFace/video_crop_face_bbox_list.txt'
    with open(file_path, 'r') as file:
        with open(out_file_path, 'w') as out_file:
            for line in file.readlines():
                lines = line.split()
                file_name = lines[0]
                bbox = lines[1]
                out_info = '{} {}\n'.format(file_name, bbox)
                out_file.write(out_info)
            os.system('chmod a+wr {}'.format(out_file_path))


def filter_video_crop_person():
    filter_file_list_path = r'/rootfs/media/yery/Kaso/data/VideoGenderCropPerson/select_video_images.txt'
    file_lable_list_path = r'/rootfs/media/yery/Kaso/data/VideoGenderCropPerson/select_video_image_labels.txt'
    face_file_list_path = r'/rootfs/media/yery/Kaso/data/VideoGenderCropFace/video_crop_face_list.attr.txt'

    filter_video_set = set()
    with open(filter_file_list_path, 'r') as file:
        for line in file.readlines():
            # gd/zs/sqqjd/yxxhjt/150100414a54443452067fa6d44b6600/video/20190920_093319/00084_000.jpg
            file_path = line.strip().split()[0]
            video_name = os.path.dirname(file_path)
            filter_video_set.add(video_name)

    video_labels = {}
    video_image_labels = []
    with open(file_lable_list_path, 'r') as file:
        for line in file.readlines():
            # gd/zs/dqjd/xgly/150100414a5444345203bcd04294b500/video/20190912_154308/00000_000.jpg 1
            lines = line.strip().split()
            file_path = lines[0]
            video_name = os.path.dirname(file_path)
            if video_name in filter_video_set:
                continue
            label = lines[1]
            # image_name = os.path.basename(file_path)
            video_image_labels.append(line)
            video_labels[video_name] = label

    out_file_dir = os.path.dirname(file_lable_list_path)
    out_file_path = os.path.join(out_file_dir, 'video_label_list.txt')
    with open(out_file_path, 'w') as file:
        video_names = list(video_labels.keys())
        video_names.sort()
        for video_name in video_names:
            label = video_labels[video_name]
            out_info = '{} {}\n'.format(video_name, label)
            file.write(out_info)
    os.system('chmod a+wr {}'.format(out_file_path))

    out_file_path = os.path.join(out_file_dir, 'video_image_person_label_list.txt')
    with open(out_file_path, 'w') as file:
        video_image_labels.sort()
        for video_image_label in video_image_labels:
            file.write(video_image_label)
    os.system('chmod a+wr {}'.format(out_file_path))

    video_image_labels = []
    with open(face_file_list_path, 'r') as file:
        for line in file.readlines():
            # gd/zs/dqjd/xgly/150100414a5444345203bcd04294b500/video/20190912_154308/00000_000.jpg
            lines = line.strip().split()
            file_path = lines[0]
            video_name = os.path.dirname(file_path)
            label = video_labels.get(video_name, None)
            if label is None:
                continue
            video_image_labels.append('{} {}\n'.format(file_path, label))

    out_file_path = os.path.join(os.path.dirname(face_file_list_path), 'video_image_face_label_list.txt')
    with open(out_file_path, 'w') as file:
        video_image_labels.sort()
        for video_image_label in video_image_labels:
            file.write(video_image_label)
    os.system('chmod a+wr {}'.format(out_file_path))


def filter_same_file():
    file_list_paths = [
        r'/rootfs/media/kasim/Data1/data/VideoGenderCropFace/big_angle_face_image_list1.txt',
        r'/rootfs/media/kasim/Data1/data/VideoGenderCropFace/big_angle_face_image_list2.txt',
    ]
    out_file_path = r'/rootfs/media/kasim/Data1/data/VideoGenderCropFace/big_angle_face_image_list.txt'

    file_set = set()
    for file_list_path in file_list_paths:
        with open(file_list_path, 'r') as file:
            for line in file.readlines():
                # gd/zs/sqqjd/yxxhjt/150100414a54443452067fa6d44b6600/video/20190920_093319/00084_000.jpg
                file_path = line.strip().split()[0]
                file_set.add(file_path)

    with open(out_file_path, 'w') as file:
        file_list = list(file_set)
        file_list.sort()
        for file_path in file_list:
            file.write(file_path + '\n')
    os.system('chmod a+wr {}'.format(out_file_path))
    print('File Count:', len(file_set))


def filter_file_feature():
    feature_file_list_path = r'/rootfs/media/kasim/Data1/data/VideoGenderCropFace/BoxFeature.txt'
    feature_file_path = r'/rootfs/media/kasim/Data1/data/VideoGenderCropFace/Feature.dat'

    # file_list_path = r'/rootfs/media/kasim/Data1/data/VideoGenderCropFace/big_angle/FaceRecSel.txt'
    # out_file_path = r'/rootfs/media/kasim/Data1/data/VideoGenderCropFace/big_angle/FeatureFileList.txt'
    # out_file2_path = r'/rootfs/media/kasim/Data1/data/VideoGenderCropFace/big_angle/FeatureFileList2.txt'
    # out_feature_path = r'/rootfs/media/kasim/Data1/data/VideoGenderCropFace/big_angle/Feature.dat'

    file_list_path = r'/rootfs/media/kasim/Data1/data/VideoGenderCropFace/small_angle/FaceRecSel.txt'
    out_file_path = r'/rootfs/media/kasim/Data1/data/VideoGenderCropFace/small_angle/FeatureFileList.txt'
    out_file2_path = r'/rootfs/media/kasim/Data1/data/VideoGenderCropFace/small_angle/FeatureFileList2.txt'
    out_feature_path = r'/rootfs/media/kasim/Data1/data/VideoGenderCropFace/small_angle/Feature.dat'

    file_list = []
    with open(file_list_path, 'r') as file:
        for line in file.readlines():
            # gd/zs/sqqjd/yxxhjt/150100414a54443452067fa6d44b6600/video/20190920_093319/00084_000.jpg
            file_path = line.strip().split()[0]
            file_list.append(file_path)

    file_features = read_features(feature_file_list_path, feature_file_path)

    out_file = open(out_file_path, 'w')
    out_file2 = open(out_file2_path, 'w')
    out_feature_file = open(out_feature_path, 'wb')

    feature_file_count = 0
    no_feature_file_count = 0
    file_list.sort()
    for file_path in file_list:
        feature = file_features.get(file_path, None)
        if feature is None:
            out_file2.write(file_path + '\n')
            no_feature_file_count += 1
        else:
            out_file.write(file_path + '\n')
            out_feature_file.write(feature)
            feature_file_count += 1

    out_file.close()
    out_file2.close()
    out_feature_file.close()
    os.system('chmod a+wr {}'.format(out_file_path))
    os.system('chmod a+wr {}'.format(out_file2_path))
    os.system('chmod a+wr {}'.format(out_feature_path))
    print('feature_file_count:', feature_file_count, 'no_feature_file_count:', no_feature_file_count)


def copy_file():
    src_file_dir = r'/rootfs/media/kasim/Data1/data/VideoGenderCropFace'

    # file_index = 41337
    # out_file_path = r'/rootfs/media/kasim/Data1/data/VideoGenderCropFace/big_angle/image_list.txt'
    # file_list_path = r'/rootfs/media/kasim/Data1/data/VideoGenderCropFace/big_angle/FaceRecSel.txt'
    # dst_file_dir = r'/rootfs/media/kasim/Data1/data/headpose/images'

    file_index = 25739
    out_file_path = r'/rootfs/media/kasim/Data1/data/VideoGenderCropFace/small_angle/image_list.txt'
    file_list_path = r'/rootfs/media/kasim/Data1/data/VideoGenderCropFace/small_angle/FaceRecSel.txt'
    dst_file_dir = r'/rootfs/media/kasim/Data1/data/headpose_small_angle/images'

    file_count = 0
    out_file = open(out_file_path, 'w')
    with open(file_list_path, 'r') as file:
        for line in file.readlines():
            file_name = line.split()[0].strip()
            src_file_path = os.path.join(src_file_dir, file_name)
            if not os.path.exists(src_file_path):
                continue
            file_index += 1
            dst_file_name = '{:06d}.jpg'.format(file_index)
            dst_file_path = os.path.join(dst_file_dir, dst_file_name)
            shutil.copy(src_file_path, dst_file_path)
            out_file.write(dst_file_name + '\n')
            file_count += 1
    out_file.close()
    os.system('chmod a+wr {}'.format(out_file_path))
    print('file_count:', file_count)


def big_angle_dataset_split():
    # file_list_path = r'/rootfs/media/kasim/Data1/data/headpose_small_angle/small_angle/FilterFaceImage_filter.txt'
    file_list_path = r'/rootfs/media/kasim/Data1/data/headpose/big_angle/FilterFaceImage_filter.txt'

    val_ratio = 0.15

    random.seed(123456)
    out_path = os.path.dirname(file_list_path)

    out_train_path = os.path.join(out_path, 'train.txt')
    out_val_path = os.path.join(out_path, 'val.txt')

    file_list = []
    with open(file_list_path, 'r') as file:
        for line in file.readlines():
            file_list.append(line)
    random.shuffle(file_list)

    file_count = len(file_list)
    val_count = int(file_count * val_ratio)
    val_file_list = file_list[0:val_count]
    train_file_list = file_list[val_count:]

    with open(out_train_path, 'w') as file:
        for file_path in train_file_list:
            file.write(file_path)

    with open(out_val_path, 'w') as file:
        for file_path in val_file_list:
            file.write(file_path)

    os.system('chmod a+wr {}'.format(out_train_path))
    os.system('chmod a+wr {}'.format(out_val_path))


def main():
    # count = 10000
    # dict0 = {}
    # list0 = []
    # list1 = list(range(1000))
    # st0 = time.time()
    # for i in range(count):
    #     list0.append(list1)
    # dt0 = time.time()
    # st1 = time.time()
    # for i in range(count):
    #     dict0[i] = list1
    # dt1 = time.time()
    #
    # st2 = time.time()
    # for i in range(count):
    #     l0 = list0.pop(0)
    # dt2 = time.time()
    # st3 = time.time()
    # for i in range(count):
    #     d0 = dict0.pop(i)
    # dt3 = time.time()
    #
    # print('List Add Time: {}ms, Dict Add Time: {}ms, List Del Time: {}ms, Dict Del Time: {}ms'.format((dt0-st0)*1000, (dt1-st1)*1000, (dt2-st2)*1000, (dt3-st3)*1000))
    # # List Add Time: 0.8139610290527344ms, Dict Add Time: 0.6308555603027344ms, List Del Time: 170.80163955688477ms, Dict Del Time: 1.1644363403320312ms

    # file_rename()
    # file_rename2()
    # face_rec_video()
    # filter_gaze_video()
    # cal_file_face_count()
    # filter_gaze_file()
    # split_face_rec_video()
    # cal_file_count()
    # copy_video_face_file()
    # check_video_face_file()
    # filter_file_list()
    # filter_attr_video()
    # image_to_video_list()
    # query_gallery_to_video_list()
    # filter_video_list()
    # file_list()
    # filter_attr_video2()
    # filter_attr_video3()
    # filter_attr_video4()
    # filter_features()
    # filter_features2()
    # filter_faceid_video()
    # filter_image_bbox()
    # split_face_rec_video2()
    # split_face_rec_feature()
    # gaze_video_statistics()
    # filter_file()
    # move_face_file()
    # gender_filter()
    # filter_video_crop_face_bbox()
    # filter_video_crop_person()
    # filter_same_file()
    # filter_file_feature()
    # copy_file()
    big_angle_dataset_split()

    '''
    1. filter_attr_video2(), filter_attr_video3()
    2. filter_features()
    3. filter_faceid_video()
    '''


if __name__ == '__main__':
    main()

